{"componentChunkName":"component---src-templates-bootcamp-js","path":"/bootcamp/bigdata/201-streaming-5/","result":{"data":{"site":{"siteMetadata":{"title":"William Blogs & More","description":"team knowledge base"}},"allBootcamp":{"edges":[{"node":{"fields":{"id":"410a853e-19db-5ecf-9193-d90fd4e2b164","slug":"/README/","title":"README"}}},{"node":{"fields":{"id":"ebd8a7a9-4f93-5f7f-89b4-7a921fc850fd","slug":"/backend/DEV/","title":"DEV"}}},{"node":{"fields":{"id":"c97bf13a-dd3f-5e0b-9f0a-9c5f08a42d57","slug":"/backend/Restful/","title":"Restful"}}},{"node":{"fields":{"id":"81e9e69e-bf6a-51e5-843d-7b4321c330f9","slug":"/backend/curl/","title":"Curl"}}},{"node":{"fields":{"id":"2aa2ab91-e4c3-59d3-ae10-2949c78bbba2","slug":"/backend/express/","title":"Express"}}},{"node":{"fields":{"id":"923aee4e-1ea6-5b5c-9dbf-7f3ed2ccd8de","slug":"/backend/flow/","title":"Flow"}}},{"node":{"fields":{"id":"735e010f-fe92-58c2-a717-44e1379ec81a","slug":"/backend/microservice/","title":"Microservice"}}},{"node":{"fields":{"id":"5257ef00-003a-505c-9681-90b156b2fdc3","slug":"/backend/mongo/","title":"Mongo"}}},{"node":{"fields":{"id":"5d7e4d14-bbe0-51e7-ac00-aed822be865d","slug":"/backend/nodejs/","title":"Nodejs"}}},{"node":{"fields":{"id":"04e0f6cd-ba6d-5bcb-a711-f978053c85a9","slug":"/backend/pg/","title":"Pg"}}},{"node":{"fields":{"id":"07933914-1441-5d9a-93b3-7b8527f32834","slug":"/backend/rabbitmq/","title":"Rabbitmq"}}},{"node":{"fields":{"id":"aeee00c3-e65a-5e57-8e26-99bdee4b5972","slug":"/backend/redis/","title":"Redis"}}},{"node":{"fields":{"id":"5efa4932-4a43-5627-a52e-d6c44ea1ce37","slug":"/backend/socket/","title":"Socket"}}},{"node":{"fields":{"id":"2546451c-541b-5d1f-b34c-1f1842685cfc","slug":"/bestitconsulting/Bootcamp/1-bootcamp/","title":"1 Bootcamp"}}},{"node":{"fields":{"id":"3ce1ed0e-ce4c-53b6-89a3-dd5dc7642647","slug":"/bestitconsulting/Bootcamp/2-release_branch/","title":"2 Release_branch"}}},{"node":{"fields":{"id":"e3f26574-c4e6-5058-8336-6e761d369455","slug":"/bestitconsulting/Bootcamp/3-coding/","title":"3 Coding"}}},{"node":{"fields":{"id":"8b752412-b3d3-5bbe-87b3-d99ef91d2759","slug":"/bestitconsulting/Bootcamp/4-testing/","title":"4 Testing"}}},{"node":{"fields":{"id":"e950ea75-d6c1-5fca-8e1c-4b4fce362fab","slug":"/bestitconsulting/Bootcamp/5-CICD/","title":"5 CICD"}}},{"node":{"fields":{"id":"739b0238-56b4-5a47-bc76-3286eadcb425","slug":"/bestitconsulting/Bootcamp/6-BA/","title":"6 BA"}}},{"node":{"fields":{"id":"bc2e6ad3-6b39-54b0-8d66-e5e295f331ee","slug":"/bestitconsulting/Bootcamp/QA/","title":"QA"}}},{"node":{"fields":{"id":"484882b7-848a-591f-80d5-24d045cb9582","slug":"/bestitconsulting/Bootcamp/README/","title":"README"}}},{"node":{"fields":{"id":"27252ae9-1261-5b83-9039-45c41697c09d","slug":"/bestitconsulting/Bootcamp/Reference/","title":"Reference"}}},{"node":{"fields":{"id":"2a271bc5-0775-5cb0-99ab-056d5c0cf09f","slug":"/bestitconsulting/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"372866f9-ffd2-5576-bbc1-415732492686","slug":"/bestitconsulting/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"5eb7bdc4-c171-52bc-bf48-f413de03ea24","slug":"/bestitconsulting/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"986b6381-1b35-5253-93d5-1294d4984202","slug":"/bestitconsulting/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"8e3b3436-48d9-57f8-9d53-71494461a76e","slug":"/bestitconsulting/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"f9b1b128-a118-5569-b2ee-b6d5c347e964","slug":"/bestitconsulting/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"17a02ec4-b391-54f1-8c60-922492f1a5eb","slug":"/bigdata/1-bigdata/","title":"1 Bigdata"}}},{"node":{"fields":{"id":"36884170-cd04-548a-9a00-77e36a4b1380","slug":"/bigdata/2-hadoop/","title":"2 Hadoop"}}},{"node":{"fields":{"id":"c9cdff49-84d1-58e6-bf25-926b7f68d84c","slug":"/bigdata/201-kafka-4/","title":"201 Kafka 4"}}},{"node":{"fields":{"id":"f2387a56-a470-54c8-94aa-182fffbab8b0","slug":"/bigdata/201-nosql-8/","title":"201 Nosql 8"}}},{"node":{"fields":{"id":"4d3d0dd0-c311-51f6-9975-80fd4c5fa752","slug":"/bigdata/201-streaming-5/","title":"201 Streaming 5"}}},{"node":{"fields":{"id":"27a22e9e-6aa2-5c4e-9edf-c06b8740246d","slug":"/bigdata/3-hdfs/","title":"3 Hdfs"}}},{"node":{"fields":{"id":"f80ad3db-5700-5e3b-a5ae-2e6f5dc0c886","slug":"/bigdata/4-devops/","title":"4 Devops"}}},{"node":{"fields":{"id":"07672f34-d70e-589c-ba8e-67e284bc1d8c","slug":"/bigdata/5-hive/","title":"5 Hive"}}},{"node":{"fields":{"id":"bf3a9766-31fd-5213-8dd9-7c38ebe7bb80","slug":"/bigdata/6-spark/","title":"6 Spark"}}},{"node":{"fields":{"id":"4dab467f-b6ea-531b-8c28-9bcef71863e4","slug":"/bigdata/7-docker/","title":"7 Docker"}}},{"node":{"fields":{"id":"7ee4ff75-d3f8-5c1a-b701-facf126f3450","slug":"/bigdata/8-streaming/","title":"8 Streaming"}}},{"node":{"fields":{"id":"6785ce66-99e5-5e00-af30-b1c32c61cb12","slug":"/bigdata/9-elasticsearch/","title":"9 Elasticsearch"}}},{"node":{"fields":{"id":"9a59f47c-4cd3-51db-9962-2ffb26b14dc4","slug":"/bigdata/hdfs-kubernetes/","title":"Hdfs Kubernetes"}}},{"node":{"fields":{"id":"819a354c-d042-54db-9466-5063d21db45f","slug":"/bigdata/kubernetes/","title":"Kubernetes"}}},{"node":{"fields":{"id":"4535d21c-0f79-5cda-b235-98658a1ffe1f","slug":"/bigdata/spark-kubernetes/","title":"Spark Kubernetes"}}},{"node":{"fields":{"id":"07d1d0ca-dc16-52b5-b911-0d128a2388eb","slug":"/bigdata2/README/","title":"README"}}},{"node":{"fields":{"id":"605faba4-a685-5a80-a80a-5130700509e3","slug":"/bigdata2/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"5b1a9498-e94f-5dba-a4fa-9aa2f3da4312","slug":"/bigdata2/editor/","title":"Editor"}}},{"node":{"fields":{"id":"6e1973a3-4045-57b9-a096-d498f5d2802b","slug":"/bigdata2/hadoop/","title":"Hadoop"}}},{"node":{"fields":{"id":"e63f21ad-65e0-56db-891e-60a89c5f93da","slug":"/bigdata2/kafka/","title":"Kafka"}}},{"node":{"fields":{"id":"2efc1fee-93bd-53eb-b5e0-672ff2acc9a4","slug":"/bigdata2/scala/","title":"Scala"}}},{"node":{"fields":{"id":"001e1d27-4cba-5926-ba33-8197e2065136","slug":"/bigdata2/spark/","title":"Spark"}}},{"node":{"fields":{"id":"2e2078b0-b734-5ae3-873c-022349c1c3cf","slug":"/bigdata2/vscode/","title":"Vscode"}}},{"node":{"fields":{"id":"b52de8e7-4791-556a-9989-b84717cd3370","slug":"/bigdata2/web-resource/","title":"Web Resource"}}},{"node":{"fields":{"id":"e0378d10-6064-560e-a6b2-cbe89a820d26","slug":"/bigdata2/zookeeper/","title":"Zookeeper"}}},{"node":{"fields":{"id":"616110f5-9073-5438-846b-ff7584d4e922","slug":"/cloud/aws/","title":"Aws"}}},{"node":{"fields":{"id":"c788368a-0f0e-5661-8fe9-c469d5a278d8","slug":"/cloud/azure/","title":"Azure"}}},{"node":{"fields":{"id":"30bec85f-fc28-5638-b42d-a952b414de7c","slug":"/cloud/elk/","title":"Elk"}}},{"node":{"fields":{"id":"57773e69-6a69-5e86-a641-af37ffe1ad40","slug":"/cloud/gcp/","title":"Gcp"}}},{"node":{"fields":{"id":"6688fe10-a20c-5479-bc80-170063d3e58c","slug":"/frontend/apollo/","title":"Apollo"}}},{"node":{"fields":{"id":"09e2c05f-9d10-5776-8fe8-44e9a565a20e","slug":"/frontend/graphql/","title":"Graphql"}}},{"node":{"fields":{"id":"28d38d20-c6e2-50ae-ac83-d3e3b9f25248","slug":"/frontend/proxy/","title":"Proxy"}}},{"node":{"fields":{"id":"ca4444ea-9796-5d40-98e1-c01d80f88221","slug":"/frontend/react-router/","title":"React Router"}}},{"node":{"fields":{"id":"61e06420-e9e4-5aa4-9d71-bf618be3f4fe","slug":"/frontend/react/","title":"React"}}},{"node":{"fields":{"id":"9d95393b-96be-5065-83bb-536b746eed21","slug":"/frontend/redux/","title":"Redux"}}},{"node":{"fields":{"id":"e3bb0f44-4819-556c-8511-7402303e36c5","slug":"/misc/0428/","title":"0428"}}},{"node":{"fields":{"id":"dfcc8a86-17af-5c8c-b831-9fe4fa51bf0e","slug":"/misc/HISTORY/","title":"HISTORY"}}},{"node":{"fields":{"id":"6f004260-a55d-5a2d-8146-d197509fd980","slug":"/misc/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"30096e9c-2593-5fc8-80d5-1f7648898888","slug":"/misc/git/","title":"Git"}}},{"node":{"fields":{"id":"bed21539-b0d7-5936-9c15-96905db68ac9","slug":"/misc/gitlab/","title":"Gitlab"}}},{"node":{"fields":{"id":"ab43544c-0ef1-56d2-a0a3-035fdd1cf7fe","slug":"/misc/installation/","title":"Installation"}}},{"node":{"fields":{"id":"b2e02e77-9623-5967-a541-fe1829315b47","slug":"/misc/misc/","title":"Misc"}}},{"node":{"fields":{"id":"4c7d96dc-fddc-52fd-9333-9fa6d2633874","slug":"/misc/vocabulary/","title":"Vocabulary"}}},{"node":{"fields":{"id":"f2a8cc1a-8b27-541c-b68e-8c5dc6d0f6b3","slug":"/poc/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"ae6e68d8-0f75-5fac-ad96-f042bddd50aa","slug":"/poc/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"689b1b72-e7b6-5b07-9890-50f653fdd7df","slug":"/poc/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"003c5b81-20aa-57aa-b268-17aaaa338e68","slug":"/poc/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"e0f53ac9-fbe0-5c72-8b69-ee2cce387f90","slug":"/poc/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"c5718e38-5cc1-566e-a649-a4d501877995","slug":"/poc/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"5a68cfb3-1b53-5527-a6ad-b8277381fab4","slug":"/poc/radar/steps/","title":"Steps"}}}]},"bootcamp":{"fields":{"id":"4d3d0dd0-c311-51f6-9975-80fd4c5fa752","title":"201 Streaming 5","slug":"/bigdata/201-streaming-5/","tag":"bigdata","category":"bootcamp","date":"2020-12-17","size":1174},"html":"

Essential concepts in Spark Streaming:

\n\n
\n

Discretized Stream (DStream) is the fundamental concept of Spark Streaming. It is basically a stream of RDDs with elements being the data received from input streams for batch (possibly extended in scope by windowed or stateful operators).

\n
\n

Spark streaming

\n\n

Spark sources

\n\n

Kafka Integration

\n

\n \n \n \n \n

\n

Reference

\n","tableOfContents":""},"previous":{"fields":{"id":"f2387a56-a470-54c8-94aa-182fffbab8b0","title":"201 Nosql 8","slug":"/bigdata/201-nosql-8/"},"excerpt":"1. RDBMS vs NoSQL 2. CAP THEOREM (上限定理) \n 3. NOSQL DB TYPES SQL/New SQL Document DB: semi-structured data Column Based DB: ETL, data…"},"next":{"fields":{"id":"27a22e9e-6aa2-5c4e-9edf-c06b8740246d","title":"3 Hdfs","slug":"/bigdata/3-hdfs/"},"excerpt":"HDFS NameNode Master, Metadata DataNodes NameNode dameon must be running at all times. If the NameNode stops, the cluster becomes…"}},"pageContext":{"id":"4d3d0dd0-c311-51f6-9975-80fd4c5fa752","prevId":"f2387a56-a470-54c8-94aa-182fffbab8b0","nextId":"27a22e9e-6aa2-5c4e-9edf-c06b8740246d"}},"staticQueryHashes":["1576573137","63159454"]}